{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 性别鉴定"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import random\n",
    "from nltk.corpus import names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 提取特征\n",
    "def gender_features(word):\n",
    "    return {'last_letter': word[-1]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_set = ([(name, 'male') for name in names.words('male.txt')] +\n",
    "            [(name, 'female') for name in names.words('female.txt')])\n",
    "random.shuffle(name_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "featuresets = [(gender_features(n), g) for (n, g) in name_set]\n",
    "train_set, test_set = featuresets[500:], featuresets[:500]\n",
    "\n",
    "classifier = nltk.NaiveBayesClassifier.train(train_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'male'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier.classify(gender_features('Neo'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'female'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier.classify(gender_features('Trinity'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.76"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.classify.accuracy(classifier, test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Informative Features\n",
      "             last_letter = 'a'            female : male   =     34.6 : 1.0\n",
      "             last_letter = 'k'              male : female =     31.7 : 1.0\n",
      "             last_letter = 'f'              male : female =     28.8 : 1.0\n",
      "             last_letter = 'p'              male : female =     12.5 : 1.0\n",
      "             last_letter = 'd'              male : female =      9.4 : 1.0\n"
     ]
    }
   ],
   "source": [
    "# 显示的比率为似然比，用于比较不同特征-结果关系\n",
    "classifier.show_most_informative_features(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 在处理大型预料库时，构建一个包含每一个实例的特征的单独的链表会使用大量的内存。\n",
    "# 下述方式不会在内容中存储所有的特征集对象\n",
    "from nltk.classify import apply_features\n",
    "train_set = apply_features(gender_features, name_set[500:])\n",
    "test_set = apply_features(gender_features, name_set[:500])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "设定训练集、开发集和测试集。训练接用于训练，开发集用于在每次训练后找出预测不准确的样例，然后分析是哪些原因，进而改进特征集。最后用测试集进行最终的验证"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 文档分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import movie_reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "documents = [(list(movie_reviews.words(fileid)), category)\n",
    "            for category in movie_reviews.categories()\n",
    "            for fileid in movie_reviews.fileids(category)]\n",
    "random.shuffle(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建整个预料库中前2000个最频繁词的链表\n",
    "all_words = nltk.FreqDist(w.lower() for w in movie_reviews.words())\n",
    "word_features = list(all_words.keys())[:2000]\n",
    "\n",
    "# 特征提取器\n",
    "def documnet_features(document):\n",
    "    document_words = set(document)\n",
    "    features = {}\n",
    "    for word in word_features:\n",
    "        features['contains(%s)' % word] = (word in document_words)\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'contains(plot)': True,\n",
       " 'contains(:)': True,\n",
       " 'contains(two)': True,\n",
       " 'contains(teen)': False,\n",
       " 'contains(couples)': False,\n",
       " 'contains(go)': False,\n",
       " 'contains(to)': True,\n",
       " 'contains(a)': True,\n",
       " 'contains(church)': False,\n",
       " 'contains(party)': False,\n",
       " 'contains(,)': True,\n",
       " 'contains(drink)': False,\n",
       " 'contains(and)': True,\n",
       " 'contains(then)': True,\n",
       " 'contains(drive)': False,\n",
       " 'contains(.)': True,\n",
       " 'contains(they)': True,\n",
       " 'contains(get)': True,\n",
       " 'contains(into)': True,\n",
       " 'contains(an)': True,\n",
       " 'contains(accident)': False,\n",
       " 'contains(one)': True,\n",
       " 'contains(of)': True,\n",
       " 'contains(the)': True,\n",
       " 'contains(guys)': False,\n",
       " 'contains(dies)': False,\n",
       " 'contains(but)': True,\n",
       " 'contains(his)': True,\n",
       " 'contains(girlfriend)': True,\n",
       " 'contains(continues)': False,\n",
       " 'contains(see)': False,\n",
       " 'contains(him)': True,\n",
       " 'contains(in)': True,\n",
       " 'contains(her)': False,\n",
       " 'contains(life)': False,\n",
       " 'contains(has)': True,\n",
       " 'contains(nightmares)': False,\n",
       " 'contains(what)': True,\n",
       " \"contains(')\": True,\n",
       " 'contains(s)': True,\n",
       " 'contains(deal)': False,\n",
       " 'contains(?)': False,\n",
       " 'contains(watch)': True,\n",
       " 'contains(movie)': True,\n",
       " 'contains(\")': True,\n",
       " 'contains(sorta)': False,\n",
       " 'contains(find)': False,\n",
       " 'contains(out)': True,\n",
       " 'contains(critique)': False,\n",
       " 'contains(mind)': False,\n",
       " 'contains(-)': True,\n",
       " 'contains(fuck)': False,\n",
       " 'contains(for)': True,\n",
       " 'contains(generation)': False,\n",
       " 'contains(that)': True,\n",
       " 'contains(touches)': False,\n",
       " 'contains(on)': True,\n",
       " 'contains(very)': True,\n",
       " 'contains(cool)': False,\n",
       " 'contains(idea)': True,\n",
       " 'contains(presents)': False,\n",
       " 'contains(it)': True,\n",
       " 'contains(bad)': False,\n",
       " 'contains(package)': False,\n",
       " 'contains(which)': True,\n",
       " 'contains(is)': True,\n",
       " 'contains(makes)': False,\n",
       " 'contains(this)': True,\n",
       " 'contains(review)': False,\n",
       " 'contains(even)': False,\n",
       " 'contains(harder)': False,\n",
       " 'contains(write)': False,\n",
       " 'contains(since)': False,\n",
       " 'contains(i)': False,\n",
       " 'contains(generally)': False,\n",
       " 'contains(applaud)': False,\n",
       " 'contains(films)': False,\n",
       " 'contains(attempt)': False,\n",
       " 'contains(break)': False,\n",
       " 'contains(mold)': False,\n",
       " 'contains(mess)': False,\n",
       " 'contains(with)': True,\n",
       " 'contains(your)': False,\n",
       " 'contains(head)': False,\n",
       " 'contains(such)': False,\n",
       " 'contains(()': True,\n",
       " 'contains(lost)': False,\n",
       " 'contains(highway)': False,\n",
       " 'contains(&)': False,\n",
       " 'contains(memento)': False,\n",
       " 'contains())': True,\n",
       " 'contains(there)': True,\n",
       " 'contains(are)': True,\n",
       " 'contains(good)': False,\n",
       " 'contains(ways)': False,\n",
       " 'contains(making)': True,\n",
       " 'contains(all)': True,\n",
       " 'contains(types)': False,\n",
       " 'contains(these)': False,\n",
       " 'contains(folks)': False,\n",
       " 'contains(just)': True,\n",
       " 'contains(didn)': False,\n",
       " 'contains(t)': False,\n",
       " 'contains(snag)': False,\n",
       " 'contains(correctly)': False,\n",
       " 'contains(seem)': False,\n",
       " 'contains(have)': True,\n",
       " 'contains(taken)': False,\n",
       " 'contains(pretty)': False,\n",
       " 'contains(neat)': False,\n",
       " 'contains(concept)': False,\n",
       " 'contains(executed)': False,\n",
       " 'contains(terribly)': False,\n",
       " 'contains(so)': False,\n",
       " 'contains(problems)': True,\n",
       " 'contains(well)': True,\n",
       " 'contains(its)': False,\n",
       " 'contains(main)': False,\n",
       " 'contains(problem)': False,\n",
       " 'contains(simply)': False,\n",
       " 'contains(too)': False,\n",
       " 'contains(jumbled)': False,\n",
       " 'contains(starts)': False,\n",
       " 'contains(off)': False,\n",
       " 'contains(normal)': False,\n",
       " 'contains(downshifts)': False,\n",
       " 'contains(fantasy)': False,\n",
       " 'contains(world)': True,\n",
       " 'contains(you)': True,\n",
       " 'contains(as)': True,\n",
       " 'contains(audience)': False,\n",
       " 'contains(member)': False,\n",
       " 'contains(no)': False,\n",
       " 'contains(going)': False,\n",
       " 'contains(dreams)': False,\n",
       " 'contains(characters)': False,\n",
       " 'contains(coming)': False,\n",
       " 'contains(back)': False,\n",
       " 'contains(from)': True,\n",
       " 'contains(dead)': False,\n",
       " 'contains(others)': True,\n",
       " 'contains(who)': True,\n",
       " 'contains(look)': True,\n",
       " 'contains(like)': True,\n",
       " 'contains(strange)': False,\n",
       " 'contains(apparitions)': False,\n",
       " 'contains(disappearances)': False,\n",
       " 'contains(looooot)': False,\n",
       " 'contains(chase)': True,\n",
       " 'contains(scenes)': False,\n",
       " 'contains(tons)': False,\n",
       " 'contains(weird)': False,\n",
       " 'contains(things)': True,\n",
       " 'contains(happen)': False,\n",
       " 'contains(most)': True,\n",
       " 'contains(not)': True,\n",
       " 'contains(explained)': False,\n",
       " 'contains(now)': False,\n",
       " 'contains(personally)': False,\n",
       " 'contains(don)': False,\n",
       " 'contains(trying)': False,\n",
       " 'contains(unravel)': False,\n",
       " 'contains(film)': False,\n",
       " 'contains(every)': False,\n",
       " 'contains(when)': True,\n",
       " 'contains(does)': False,\n",
       " 'contains(give)': False,\n",
       " 'contains(me)': True,\n",
       " 'contains(same)': True,\n",
       " 'contains(clue)': False,\n",
       " 'contains(over)': False,\n",
       " 'contains(again)': False,\n",
       " 'contains(kind)': True,\n",
       " 'contains(fed)': False,\n",
       " 'contains(up)': False,\n",
       " 'contains(after)': False,\n",
       " 'contains(while)': True,\n",
       " 'contains(biggest)': False,\n",
       " 'contains(obviously)': False,\n",
       " 'contains(got)': True,\n",
       " 'contains(big)': False,\n",
       " 'contains(secret)': False,\n",
       " 'contains(hide)': False,\n",
       " 'contains(seems)': False,\n",
       " 'contains(want)': False,\n",
       " 'contains(completely)': False,\n",
       " 'contains(until)': False,\n",
       " 'contains(final)': False,\n",
       " 'contains(five)': False,\n",
       " 'contains(minutes)': False,\n",
       " 'contains(do)': True,\n",
       " 'contains(make)': True,\n",
       " 'contains(entertaining)': False,\n",
       " 'contains(thrilling)': False,\n",
       " 'contains(or)': False,\n",
       " 'contains(engaging)': False,\n",
       " 'contains(meantime)': False,\n",
       " 'contains(really)': False,\n",
       " 'contains(sad)': False,\n",
       " 'contains(part)': False,\n",
       " 'contains(arrow)': False,\n",
       " 'contains(both)': False,\n",
       " 'contains(dig)': False,\n",
       " 'contains(flicks)': False,\n",
       " 'contains(we)': False,\n",
       " 'contains(actually)': True,\n",
       " 'contains(figured)': False,\n",
       " 'contains(by)': True,\n",
       " 'contains(half)': False,\n",
       " 'contains(way)': True,\n",
       " 'contains(point)': False,\n",
       " 'contains(strangeness)': False,\n",
       " 'contains(did)': False,\n",
       " 'contains(start)': True,\n",
       " 'contains(little)': True,\n",
       " 'contains(bit)': False,\n",
       " 'contains(sense)': False,\n",
       " 'contains(still)': False,\n",
       " 'contains(more)': False,\n",
       " 'contains(guess)': False,\n",
       " 'contains(bottom)': False,\n",
       " 'contains(line)': False,\n",
       " 'contains(movies)': True,\n",
       " 'contains(should)': False,\n",
       " 'contains(always)': False,\n",
       " 'contains(sure)': False,\n",
       " 'contains(before)': False,\n",
       " 'contains(given)': False,\n",
       " 'contains(password)': False,\n",
       " 'contains(enter)': False,\n",
       " 'contains(understanding)': False,\n",
       " 'contains(mean)': False,\n",
       " 'contains(showing)': False,\n",
       " 'contains(melissa)': False,\n",
       " 'contains(sagemiller)': False,\n",
       " 'contains(running)': False,\n",
       " 'contains(away)': False,\n",
       " 'contains(visions)': False,\n",
       " 'contains(about)': True,\n",
       " 'contains(20)': False,\n",
       " 'contains(throughout)': False,\n",
       " 'contains(plain)': False,\n",
       " 'contains(lazy)': False,\n",
       " 'contains(!)': True,\n",
       " 'contains(okay)': False,\n",
       " 'contains(people)': False,\n",
       " 'contains(chasing)': False,\n",
       " 'contains(know)': False,\n",
       " 'contains(need)': False,\n",
       " 'contains(how)': True,\n",
       " 'contains(giving)': False,\n",
       " 'contains(us)': True,\n",
       " 'contains(different)': False,\n",
       " 'contains(offering)': False,\n",
       " 'contains(further)': False,\n",
       " 'contains(insight)': False,\n",
       " 'contains(down)': False,\n",
       " 'contains(apparently)': False,\n",
       " 'contains(studio)': False,\n",
       " 'contains(took)': False,\n",
       " 'contains(director)': False,\n",
       " 'contains(chopped)': False,\n",
       " 'contains(themselves)': False,\n",
       " 'contains(shows)': False,\n",
       " 'contains(might)': False,\n",
       " 'contains(ve)': False,\n",
       " 'contains(been)': False,\n",
       " 'contains(decent)': False,\n",
       " 'contains(here)': True,\n",
       " 'contains(somewhere)': False,\n",
       " 'contains(suits)': False,\n",
       " 'contains(decided)': False,\n",
       " 'contains(turning)': False,\n",
       " 'contains(music)': False,\n",
       " 'contains(video)': False,\n",
       " 'contains(edge)': False,\n",
       " 'contains(would)': False,\n",
       " 'contains(actors)': False,\n",
       " 'contains(although)': False,\n",
       " 'contains(wes)': False,\n",
       " 'contains(bentley)': False,\n",
       " 'contains(seemed)': False,\n",
       " 'contains(be)': True,\n",
       " 'contains(playing)': True,\n",
       " 'contains(exact)': False,\n",
       " 'contains(character)': False,\n",
       " 'contains(he)': True,\n",
       " 'contains(american)': False,\n",
       " 'contains(beauty)': False,\n",
       " 'contains(only)': True,\n",
       " 'contains(new)': False,\n",
       " 'contains(neighborhood)': False,\n",
       " 'contains(my)': False,\n",
       " 'contains(kudos)': False,\n",
       " 'contains(holds)': False,\n",
       " 'contains(own)': True,\n",
       " 'contains(entire)': False,\n",
       " 'contains(feeling)': False,\n",
       " 'contains(unraveling)': False,\n",
       " 'contains(overall)': False,\n",
       " 'contains(doesn)': False,\n",
       " 'contains(stick)': False,\n",
       " 'contains(because)': False,\n",
       " 'contains(entertain)': False,\n",
       " 'contains(confusing)': False,\n",
       " 'contains(rarely)': False,\n",
       " 'contains(excites)': False,\n",
       " 'contains(feels)': False,\n",
       " 'contains(redundant)': False,\n",
       " 'contains(runtime)': False,\n",
       " 'contains(despite)': False,\n",
       " 'contains(ending)': False,\n",
       " 'contains(explanation)': False,\n",
       " 'contains(craziness)': False,\n",
       " 'contains(came)': False,\n",
       " 'contains(oh)': False,\n",
       " 'contains(horror)': False,\n",
       " 'contains(slasher)': False,\n",
       " 'contains(flick)': False,\n",
       " 'contains(packaged)': False,\n",
       " 'contains(someone)': False,\n",
       " 'contains(assuming)': False,\n",
       " 'contains(genre)': False,\n",
       " 'contains(hot)': False,\n",
       " 'contains(kids)': False,\n",
       " 'contains(also)': True,\n",
       " 'contains(wrapped)': False,\n",
       " 'contains(production)': False,\n",
       " 'contains(years)': False,\n",
       " 'contains(ago)': False,\n",
       " 'contains(sitting)': False,\n",
       " 'contains(shelves)': False,\n",
       " 'contains(ever)': True,\n",
       " 'contains(whatever)': False,\n",
       " 'contains(skip)': False,\n",
       " 'contains(where)': True,\n",
       " 'contains(joblo)': False,\n",
       " 'contains(nightmare)': False,\n",
       " 'contains(elm)': False,\n",
       " 'contains(street)': False,\n",
       " 'contains(3)': False,\n",
       " 'contains(7)': False,\n",
       " 'contains(/)': False,\n",
       " 'contains(10)': False,\n",
       " 'contains(blair)': False,\n",
       " 'contains(witch)': False,\n",
       " 'contains(2)': False,\n",
       " 'contains(crow)': False,\n",
       " 'contains(9)': False,\n",
       " 'contains(salvation)': False,\n",
       " 'contains(4)': False,\n",
       " 'contains(stir)': False,\n",
       " 'contains(echoes)': False,\n",
       " 'contains(8)': False,\n",
       " 'contains(happy)': False,\n",
       " 'contains(bastard)': False,\n",
       " 'contains(quick)': True,\n",
       " 'contains(damn)': False,\n",
       " 'contains(y2k)': False,\n",
       " 'contains(bug)': False,\n",
       " 'contains(starring)': False,\n",
       " 'contains(jamie)': False,\n",
       " 'contains(lee)': False,\n",
       " 'contains(curtis)': False,\n",
       " 'contains(another)': False,\n",
       " 'contains(baldwin)': False,\n",
       " 'contains(brother)': False,\n",
       " 'contains(william)': False,\n",
       " 'contains(time)': False,\n",
       " 'contains(story)': False,\n",
       " 'contains(regarding)': False,\n",
       " 'contains(crew)': False,\n",
       " 'contains(tugboat)': False,\n",
       " 'contains(comes)': False,\n",
       " 'contains(across)': False,\n",
       " 'contains(deserted)': False,\n",
       " 'contains(russian)': False,\n",
       " 'contains(tech)': False,\n",
       " 'contains(ship)': False,\n",
       " 'contains(kick)': False,\n",
       " 'contains(power)': False,\n",
       " 'contains(within)': False,\n",
       " 'contains(gore)': False,\n",
       " 'contains(bringing)': False,\n",
       " 'contains(few)': False,\n",
       " 'contains(action)': True,\n",
       " 'contains(sequences)': False,\n",
       " 'contains(virus)': False,\n",
       " 'contains(empty)': False,\n",
       " 'contains(flash)': False,\n",
       " 'contains(substance)': False,\n",
       " 'contains(why)': False,\n",
       " 'contains(was)': False,\n",
       " 'contains(middle)': False,\n",
       " 'contains(nowhere)': False,\n",
       " 'contains(origin)': False,\n",
       " 'contains(pink)': False,\n",
       " 'contains(flashy)': False,\n",
       " 'contains(thing)': False,\n",
       " 'contains(hit)': False,\n",
       " 'contains(mir)': False,\n",
       " 'contains(course)': True,\n",
       " 'contains(donald)': False,\n",
       " 'contains(sutherland)': False,\n",
       " 'contains(stumbling)': False,\n",
       " 'contains(around)': False,\n",
       " 'contains(drunkenly)': False,\n",
       " 'contains(hey)': False,\n",
       " 'contains(let)': False,\n",
       " 'contains(some)': False,\n",
       " 'contains(robots)': False,\n",
       " 'contains(acting)': False,\n",
       " 'contains(below)': False,\n",
       " 'contains(average)': False,\n",
       " 'contains(likes)': False,\n",
       " 'contains(re)': True,\n",
       " 'contains(likely)': False,\n",
       " 'contains(work)': False,\n",
       " 'contains(halloween)': False,\n",
       " 'contains(h20)': False,\n",
       " 'contains(wasted)': False,\n",
       " 'contains(real)': False,\n",
       " 'contains(star)': False,\n",
       " 'contains(stan)': False,\n",
       " 'contains(winston)': False,\n",
       " 'contains(robot)': False,\n",
       " 'contains(design)': False,\n",
       " 'contains(schnazzy)': False,\n",
       " 'contains(cgi)': False,\n",
       " 'contains(occasional)': False,\n",
       " 'contains(shot)': False,\n",
       " 'contains(picking)': False,\n",
       " 'contains(brain)': False,\n",
       " 'contains(if)': True,\n",
       " 'contains(body)': False,\n",
       " 'contains(parts)': False,\n",
       " 'contains(turn)': False,\n",
       " 'contains(otherwise)': False,\n",
       " 'contains(much)': False,\n",
       " 'contains(sunken)': False,\n",
       " 'contains(jaded)': False,\n",
       " 'contains(viewer)': False,\n",
       " 'contains(thankful)': False,\n",
       " 'contains(invention)': False,\n",
       " 'contains(timex)': False,\n",
       " 'contains(indiglo)': False,\n",
       " 'contains(based)': False,\n",
       " 'contains(late)': False,\n",
       " 'contains(1960)': False,\n",
       " 'contains(television)': False,\n",
       " 'contains(show)': False,\n",
       " 'contains(name)': False,\n",
       " 'contains(mod)': False,\n",
       " 'contains(squad)': False,\n",
       " 'contains(tells)': False,\n",
       " 'contains(tale)': False,\n",
       " 'contains(three)': False,\n",
       " 'contains(reformed)': False,\n",
       " 'contains(criminals)': False,\n",
       " 'contains(under)': False,\n",
       " 'contains(employ)': False,\n",
       " 'contains(police)': False,\n",
       " 'contains(undercover)': True,\n",
       " 'contains(however)': True,\n",
       " 'contains(wrong)': True,\n",
       " 'contains(evidence)': False,\n",
       " 'contains(gets)': True,\n",
       " 'contains(stolen)': False,\n",
       " 'contains(immediately)': False,\n",
       " 'contains(suspicion)': False,\n",
       " 'contains(ads)': False,\n",
       " 'contains(cuts)': False,\n",
       " 'contains(claire)': False,\n",
       " 'contains(dane)': False,\n",
       " 'contains(nice)': False,\n",
       " 'contains(hair)': False,\n",
       " 'contains(cute)': False,\n",
       " 'contains(outfits)': False,\n",
       " 'contains(car)': False,\n",
       " 'contains(chases)': False,\n",
       " 'contains(stuff)': False,\n",
       " 'contains(blowing)': False,\n",
       " 'contains(sounds)': False,\n",
       " 'contains(first)': False,\n",
       " 'contains(fifteen)': False,\n",
       " 'contains(quickly)': False,\n",
       " 'contains(becomes)': False,\n",
       " 'contains(apparent)': False,\n",
       " 'contains(certainly)': False,\n",
       " 'contains(slick)': False,\n",
       " 'contains(looking)': False,\n",
       " 'contains(complete)': False,\n",
       " 'contains(costumes)': False,\n",
       " 'contains(isn)': False,\n",
       " 'contains(enough)': False,\n",
       " 'contains(best)': True,\n",
       " 'contains(described)': False,\n",
       " 'contains(cross)': False,\n",
       " 'contains(between)': True,\n",
       " 'contains(hour)': False,\n",
       " 'contains(long)': False,\n",
       " 'contains(cop)': False,\n",
       " 'contains(stretched)': False,\n",
       " 'contains(span)': False,\n",
       " 'contains(single)': False,\n",
       " 'contains(clich)': False,\n",
       " 'contains(matter)': False,\n",
       " 'contains(elements)': False,\n",
       " 'contains(recycled)': False,\n",
       " 'contains(everything)': True,\n",
       " 'contains(already)': False,\n",
       " 'contains(seen)': False,\n",
       " 'contains(nothing)': False,\n",
       " 'contains(spectacular)': False,\n",
       " 'contains(sometimes)': False,\n",
       " 'contains(bordering)': False,\n",
       " 'contains(wooden)': False,\n",
       " 'contains(danes)': False,\n",
       " 'contains(omar)': False,\n",
       " 'contains(epps)': False,\n",
       " 'contains(deliver)': False,\n",
       " 'contains(their)': False,\n",
       " 'contains(lines)': False,\n",
       " 'contains(bored)': False,\n",
       " 'contains(transfers)': False,\n",
       " 'contains(onto)': False,\n",
       " 'contains(escape)': False,\n",
       " 'contains(relatively)': False,\n",
       " 'contains(unscathed)': False,\n",
       " 'contains(giovanni)': False,\n",
       " 'contains(ribisi)': False,\n",
       " 'contains(plays)': False,\n",
       " 'contains(resident)': False,\n",
       " 'contains(crazy)': False,\n",
       " 'contains(man)': False,\n",
       " 'contains(ultimately)': False,\n",
       " 'contains(being)': False,\n",
       " 'contains(worth)': True,\n",
       " 'contains(watching)': False,\n",
       " 'contains(unfortunately)': False,\n",
       " 'contains(save)': False,\n",
       " 'contains(convoluted)': False,\n",
       " 'contains(apart)': False,\n",
       " 'contains(occupying)': False,\n",
       " 'contains(screen)': True,\n",
       " 'contains(young)': False,\n",
       " 'contains(cast)': False,\n",
       " 'contains(clothes)': False,\n",
       " 'contains(hip)': False,\n",
       " 'contains(soundtrack)': False,\n",
       " 'contains(appears)': False,\n",
       " 'contains(geared)': False,\n",
       " 'contains(towards)': False,\n",
       " 'contains(teenage)': False,\n",
       " 'contains(mindset)': False,\n",
       " 'contains(r)': False,\n",
       " 'contains(rating)': False,\n",
       " 'contains(content)': False,\n",
       " 'contains(justify)': False,\n",
       " 'contains(juvenile)': False,\n",
       " 'contains(older)': False,\n",
       " 'contains(information)': False,\n",
       " 'contains(literally)': False,\n",
       " 'contains(spoon)': False,\n",
       " 'contains(hard)': False,\n",
       " 'contains(instead)': False,\n",
       " 'contains(telling)': False,\n",
       " 'contains(dialogue)': False,\n",
       " 'contains(poorly)': False,\n",
       " 'contains(written)': False,\n",
       " 'contains(extremely)': False,\n",
       " 'contains(predictable)': False,\n",
       " 'contains(progresses)': False,\n",
       " 'contains(won)': False,\n",
       " 'contains(care)': False,\n",
       " 'contains(heroes)': False,\n",
       " 'contains(any)': False,\n",
       " 'contains(jeopardy)': False,\n",
       " 'contains(ll)': False,\n",
       " 'contains(aren)': False,\n",
       " 'contains(basing)': False,\n",
       " 'contains(nobody)': False,\n",
       " 'contains(remembers)': False,\n",
       " 'contains(questionable)': False,\n",
       " 'contains(wisdom)': False,\n",
       " 'contains(especially)': True,\n",
       " 'contains(considers)': False,\n",
       " 'contains(target)': False,\n",
       " 'contains(fact)': False,\n",
       " 'contains(number)': False,\n",
       " 'contains(memorable)': False,\n",
       " 'contains(can)': False,\n",
       " 'contains(counted)': False,\n",
       " 'contains(hand)': False,\n",
       " 'contains(missing)': False,\n",
       " 'contains(finger)': False,\n",
       " 'contains(times)': False,\n",
       " 'contains(checked)': False,\n",
       " 'contains(six)': False,\n",
       " 'contains(clear)': False,\n",
       " 'contains(indication)': False,\n",
       " 'contains(them)': True,\n",
       " 'contains(than)': False,\n",
       " 'contains(cash)': False,\n",
       " 'contains(spending)': False,\n",
       " 'contains(dollar)': False,\n",
       " 'contains(judging)': False,\n",
       " 'contains(rash)': False,\n",
       " 'contains(awful)': False,\n",
       " 'contains(seeing)': True,\n",
       " 'contains(avoid)': False,\n",
       " 'contains(at)': False,\n",
       " 'contains(costs)': False,\n",
       " 'contains(quest)': False,\n",
       " 'contains(camelot)': False,\n",
       " 'contains(warner)': False,\n",
       " 'contains(bros)': False,\n",
       " 'contains(feature)': False,\n",
       " 'contains(length)': False,\n",
       " 'contains(fully)': False,\n",
       " 'contains(animated)': False,\n",
       " 'contains(steal)': False,\n",
       " 'contains(clout)': False,\n",
       " 'contains(disney)': False,\n",
       " 'contains(cartoon)': False,\n",
       " 'contains(empire)': False,\n",
       " 'contains(mouse)': False,\n",
       " 'contains(reason)': False,\n",
       " 'contains(worried)': False,\n",
       " 'contains(other)': True,\n",
       " 'contains(recent)': False,\n",
       " 'contains(challenger)': False,\n",
       " 'contains(throne)': False,\n",
       " 'contains(last)': False,\n",
       " 'contains(fall)': False,\n",
       " 'contains(promising)': False,\n",
       " 'contains(flawed)': False,\n",
       " 'contains(20th)': False,\n",
       " 'contains(century)': False,\n",
       " 'contains(fox)': False,\n",
       " 'contains(anastasia)': False,\n",
       " 'contains(hercules)': False,\n",
       " 'contains(lively)': False,\n",
       " 'contains(colorful)': False,\n",
       " 'contains(palate)': False,\n",
       " 'contains(had)': False,\n",
       " 'contains(beat)': False,\n",
       " 'contains(hands)': False,\n",
       " 'contains(crown)': False,\n",
       " 'contains(1997)': False,\n",
       " 'contains(piece)': False,\n",
       " 'contains(animation)': False,\n",
       " 'contains(year)': False,\n",
       " 'contains(contest)': False,\n",
       " 'contains(arrival)': False,\n",
       " 'contains(magic)': False,\n",
       " 'contains(kingdom)': False,\n",
       " 'contains(mediocre)': False,\n",
       " 'contains(--)': True,\n",
       " 'contains(d)': False,\n",
       " 'contains(pocahontas)': False,\n",
       " 'contains(those)': False,\n",
       " 'contains(keeping)': False,\n",
       " 'contains(score)': False,\n",
       " 'contains(nearly)': False,\n",
       " 'contains(dull)': False,\n",
       " 'contains(revolves)': False,\n",
       " 'contains(adventures)': False,\n",
       " 'contains(free)': False,\n",
       " 'contains(spirited)': False,\n",
       " 'contains(kayley)': False,\n",
       " 'contains(voiced)': False,\n",
       " 'contains(jessalyn)': False,\n",
       " 'contains(gilsig)': False,\n",
       " 'contains(early)': True,\n",
       " 'contains(daughter)': False,\n",
       " 'contains(belated)': False,\n",
       " 'contains(knight)': False,\n",
       " 'contains(king)': False,\n",
       " 'contains(arthur)': False,\n",
       " 'contains(round)': False,\n",
       " 'contains(table)': False,\n",
       " 'contains(dream)': False,\n",
       " 'contains(follow)': False,\n",
       " 'contains(father)': False,\n",
       " 'contains(footsteps)': False,\n",
       " 'contains(she)': True,\n",
       " 'contains(chance)': False,\n",
       " 'contains(evil)': False,\n",
       " 'contains(warlord)': False,\n",
       " 'contains(ruber)': False,\n",
       " 'contains(gary)': False,\n",
       " 'contains(oldman)': False,\n",
       " 'contains(ex)': False,\n",
       " 'contains(gone)': False,\n",
       " 'contains(steals)': False,\n",
       " 'contains(magical)': False,\n",
       " 'contains(sword)': False,\n",
       " 'contains(excalibur)': False,\n",
       " 'contains(accidentally)': False,\n",
       " 'contains(loses)': False,\n",
       " 'contains(dangerous)': True,\n",
       " 'contains(booby)': False,\n",
       " 'contains(trapped)': False,\n",
       " 'contains(forest)': False,\n",
       " 'contains(help)': True,\n",
       " 'contains(hunky)': False,\n",
       " 'contains(blind)': False,\n",
       " 'contains(timberland)': False,\n",
       " 'contains(dweller)': False,\n",
       " 'contains(garrett)': False,\n",
       " 'contains(carey)': False,\n",
       " 'contains(elwes)': False,\n",
       " 'contains(headed)': False,\n",
       " 'contains(dragon)': False,\n",
       " 'contains(eric)': False,\n",
       " 'contains(idle)': False,\n",
       " 'contains(rickles)': False,\n",
       " 'contains(arguing)': False,\n",
       " 'contains(itself)': False,\n",
       " 'contains(able)': False,\n",
       " 'contains(medieval)': False,\n",
       " 'contains(sexist)': False,\n",
       " 'contains(prove)': False,\n",
       " 'contains(fighter)': False,\n",
       " 'contains(side)': False,\n",
       " 'contains(pure)': False,\n",
       " 'contains(showmanship)': False,\n",
       " 'contains(essential)': False,\n",
       " 'contains(element)': False,\n",
       " 'contains(expected)': False,\n",
       " 'contains(climb)': False,\n",
       " 'contains(high)': False,\n",
       " 'contains(ranks)': False,\n",
       " 'contains(differentiates)': False,\n",
       " 'contains(something)': False,\n",
       " 'contains(saturday)': False,\n",
       " 'contains(morning)': False,\n",
       " 'contains(subpar)': False,\n",
       " 'contains(instantly)': False,\n",
       " 'contains(forgettable)': False,\n",
       " 'contains(songs)': False,\n",
       " 'contains(integrated)': False,\n",
       " 'contains(computerized)': False,\n",
       " 'contains(footage)': False,\n",
       " 'contains(compare)': False,\n",
       " 'contains(run)': False,\n",
       " 'contains(angry)': False,\n",
       " 'contains(ogre)': False,\n",
       " 'contains(herc)': False,\n",
       " 'contains(battle)': False,\n",
       " 'contains(hydra)': False,\n",
       " 'contains(rest)': False,\n",
       " 'contains(case)': False,\n",
       " 'contains(stink)': False,\n",
       " 'contains(none)': False,\n",
       " 'contains(remotely)': False,\n",
       " 'contains(interesting)': False,\n",
       " 'contains(race)': False,\n",
       " 'contains(bland)': False,\n",
       " 'contains(end)': False,\n",
       " 'contains(tie)': False,\n",
       " 'contains(win)': False,\n",
       " 'contains(comedy)': True,\n",
       " 'contains(shtick)': False,\n",
       " 'contains(awfully)': False,\n",
       " 'contains(cloying)': False,\n",
       " 'contains(least)': True,\n",
       " 'contains(signs)': False,\n",
       " 'contains(pulse)': False,\n",
       " 'contains(fans)': False,\n",
       " \"contains(-')\": False,\n",
       " 'contains(90s)': False,\n",
       " 'contains(tgif)': False,\n",
       " 'contains(will)': True,\n",
       " 'contains(thrilled)': False,\n",
       " 'contains(jaleel)': False,\n",
       " 'contains(urkel)': False,\n",
       " 'contains(white)': False,\n",
       " 'contains(bronson)': False,\n",
       " 'contains(balki)': False,\n",
       " 'contains(pinchot)': False,\n",
       " 'contains(sharing)': False,\n",
       " 'contains(nicely)': False,\n",
       " 'contains(realized)': False,\n",
       " 'contains(though)': False,\n",
       " 'contains(m)': False,\n",
       " 'contains(loss)': False,\n",
       " 'contains(recall)': False,\n",
       " 'contains(specific)': False,\n",
       " 'contains(providing)': False,\n",
       " 'contains(voice)': False,\n",
       " 'contains(talent)': False,\n",
       " 'contains(enthusiastic)': False,\n",
       " 'contains(paired)': False,\n",
       " 'contains(singers)': False,\n",
       " 'contains(sound)': False,\n",
       " 'contains(musical)': False,\n",
       " 'contains(moments)': False,\n",
       " 'contains(jane)': False,\n",
       " 'contains(seymour)': False,\n",
       " 'contains(celine)': False,\n",
       " 'contains(dion)': False,\n",
       " 'contains(must)': False,\n",
       " 'contains(strain)': False,\n",
       " 'contains(through)': False,\n",
       " 'contains(aside)': False,\n",
       " 'contains(children)': False,\n",
       " 'contains(probably)': False,\n",
       " 'contains(adults)': False,\n",
       " 'contains(grievous)': False,\n",
       " 'contains(error)': False,\n",
       " 'contains(lack)': False,\n",
       " 'contains(personality)': False,\n",
       " 'contains(learn)': False,\n",
       " 'contains(goes)': False,\n",
       " 'contains(synopsis)': False,\n",
       " 'contains(mentally)': False,\n",
       " 'contains(unstable)': False,\n",
       " 'contains(undergoing)': False,\n",
       " 'contains(psychotherapy)': False,\n",
       " 'contains(saves)': False,\n",
       " 'contains(boy)': False,\n",
       " 'contains(potentially)': False,\n",
       " 'contains(fatal)': False,\n",
       " 'contains(falls)': False,\n",
       " 'contains(love)': False,\n",
       " 'contains(mother)': False,\n",
       " 'contains(fledgling)': False,\n",
       " 'contains(restauranteur)': False,\n",
       " 'contains(unsuccessfully)': False,\n",
       " 'contains(attempting)': False,\n",
       " 'contains(gain)': False,\n",
       " 'contains(woman)': True,\n",
       " 'contains(favor)': False,\n",
       " 'contains(takes)': False,\n",
       " 'contains(pictures)': False,\n",
       " 'contains(kills)': False,\n",
       " 'contains(comments)': True,\n",
       " 'contains(stalked)': False,\n",
       " 'contains(yet)': False,\n",
       " 'contains(seemingly)': False,\n",
       " 'contains(endless)': True,\n",
       " 'contains(string)': False,\n",
       " 'contains(spurned)': False,\n",
       " 'contains(psychos)': False,\n",
       " 'contains(getting)': True,\n",
       " 'contains(revenge)': False,\n",
       " 'contains(type)': False,\n",
       " 'contains(stable)': False,\n",
       " 'contains(category)': False,\n",
       " 'contains(1990s)': False,\n",
       " 'contains(industry)': False,\n",
       " 'contains(theatrical)': False,\n",
       " 'contains(direct)': False,\n",
       " 'contains(proliferation)': False,\n",
       " 'contains(may)': False,\n",
       " 'contains(due)': False,\n",
       " 'contains(typically)': False,\n",
       " 'contains(inexpensive)': False,\n",
       " 'contains(produce)': False,\n",
       " 'contains(special)': False,\n",
       " 'contains(effects)': False,\n",
       " 'contains(stars)': False,\n",
       " 'contains(serve)': False,\n",
       " 'contains(vehicles)': False,\n",
       " 'contains(nudity)': False,\n",
       " 'contains(allowing)': False,\n",
       " 'contains(frequent)': False,\n",
       " 'contains(night)': False,\n",
       " 'contains(cable)': False,\n",
       " 'contains(wavers)': False,\n",
       " 'contains(slightly)': False,\n",
       " 'contains(norm)': False,\n",
       " 'contains(respect)': False,\n",
       " 'contains(psycho)': False,\n",
       " 'contains(never)': True,\n",
       " 'contains(affair)': False,\n",
       " 'contains(;)': False,\n",
       " 'contains(contrary)': False,\n",
       " 'contains(rejected)': False,\n",
       " 'contains(rather)': False,\n",
       " 'contains(lover)': False,\n",
       " 'contains(wife)': True,\n",
       " 'contains(husband)': False,\n",
       " 'contains(entry)': False,\n",
       " 'contains(doomed)': False,\n",
       " 'contains(collect)': False,\n",
       " 'contains(dust)': False,\n",
       " 'contains(viewed)': False,\n",
       " 'contains(midnight)': False,\n",
       " 'contains(provide)': False,\n",
       " 'contains(suspense)': False,\n",
       " 'contains(sets)': False,\n",
       " 'contains(interspersed)': False,\n",
       " 'contains(opening)': False,\n",
       " 'contains(credits)': False,\n",
       " 'contains(instance)': False,\n",
       " 'contains(serious)': False,\n",
       " 'contains(sounding)': False,\n",
       " 'contains(narrator)': False,\n",
       " 'contains(spouts)': False,\n",
       " 'contains(statistics)': False,\n",
       " 'contains(stalkers)': False,\n",
       " 'contains(ponders)': False,\n",
       " 'contains(cause)': False,\n",
       " 'contains(stalk)': False,\n",
       " 'contains(implicitly)': False,\n",
       " 'contains(implied)': False,\n",
       " 'contains(men)': False,\n",
       " 'contains(shown)': False,\n",
       " 'contains(snapshot)': False,\n",
       " 'contains(actor)': False,\n",
       " 'contains(jay)': False,\n",
       " 'contains(underwood)': False,\n",
       " 'contains(states)': False,\n",
       " 'contains(daryl)': False,\n",
       " 'contains(gleason)': False,\n",
       " 'contains(stalker)': False,\n",
       " 'contains(brooke)': False,\n",
       " 'contains(daniels)': False,\n",
       " 'contains(meant)': False,\n",
       " 'contains(called)': False,\n",
       " 'contains(guesswork)': False,\n",
       " 'contains(required)': False,\n",
       " 'contains(proceeds)': False,\n",
       " 'contains(begins)': False,\n",
       " 'contains(obvious)': False,\n",
       " 'contains(sequence)': False,\n",
       " 'contains(contrived)': False,\n",
       " 'contains(quite)': False,\n",
       " 'contains(brings)': False,\n",
       " 'contains(victim)': False,\n",
       " 'contains(together)': False,\n",
       " 'contains(obsesses)': False,\n",
       " 'contains(follows)': False,\n",
       " 'contains(tries)': True,\n",
       " 'contains(woo)': False,\n",
       " 'contains(plans)': False,\n",
       " 'contains(become)': False,\n",
       " 'contains(desperate)': False,\n",
       " 'contains(elaborate)': False,\n",
       " 'contains(include)': False,\n",
       " 'contains(cliche)': False,\n",
       " 'contains(murdered)': False,\n",
       " 'contains(pet)': False,\n",
       " 'contains(require)': False,\n",
       " 'contains(found)': False,\n",
       " 'contains(exception)': False,\n",
       " 'contains(cat)': False,\n",
       " 'contains(shower)': False,\n",
       " 'contains(events)': False,\n",
       " 'contains(lead)': True,\n",
       " 'contains(inevitable)': False,\n",
       " 'contains(showdown)': False,\n",
       " 'contains(survives)': False,\n",
       " 'contains(invariably)': False,\n",
       " 'contains(conclusion)': False,\n",
       " 'contains(turkey)': False,\n",
       " 'contains(uniformly)': False,\n",
       " 'contains(adequate)': False,\n",
       " 'contains(anything)': False,\n",
       " 'contains(home)': False,\n",
       " 'contains(either)': False,\n",
       " 'contains(turns)': False,\n",
       " 'contains(toward)': False,\n",
       " 'contains(melodrama)': False,\n",
       " 'contains(overdoes)': False,\n",
       " 'contains(words)': False,\n",
       " 'contains(manages)': False,\n",
       " 'contains(creepy)': False,\n",
       " 'contains(pass)': False,\n",
       " 'contains(demands)': False,\n",
       " 'contains(maryam)': False,\n",
       " 'contains(abo)': False,\n",
       " 'contains(close)': False,\n",
       " 'contains(played)': True,\n",
       " 'contains(bond)': False,\n",
       " 'contains(chick)': False,\n",
       " 'contains(living)': False,\n",
       " 'contains(daylights)': False,\n",
       " 'contains(equally)': False,\n",
       " 'contains(title)': False,\n",
       " 'contains(ditzy)': False,\n",
       " 'contains(strong)': False,\n",
       " 'contains(independent)': False,\n",
       " 'contains(business)': False,\n",
       " 'contains(owner)': False,\n",
       " 'contains(needs)': False,\n",
       " 'contains(proceed)': False,\n",
       " 'contains(example)': False,\n",
       " 'contains(suspicions)': False,\n",
       " 'contains(ensure)': False,\n",
       " 'contains(use)': False,\n",
       " 'contains(excuse)': False,\n",
       " 'contains(decides)': False,\n",
       " 'contains(return)': False,\n",
       " 'contains(toolbox)': False,\n",
       " 'contains(left)': False,\n",
       " 'contains(place)': True,\n",
       " ...}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "documnet_features(movie_reviews.words('pos/cv957_8737.txt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.81"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "featuresets = [(documnet_features(d), c) for (d, c) in documents]\n",
    "train_set, test_set = featuresets[100:], featuresets[:100]\n",
    "classifier = nltk.NaiveBayesClassifier.train(train_set)\n",
    "\n",
    "nltk.classify.accuracy(classifier, test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most Informative Features\n",
      "     contains(atrocious) = True              neg : pos    =     11.7 : 1.0\n",
      "    contains(schumacher) = True              neg : pos    =     11.7 : 1.0\n",
      "          contains(mena) = True              neg : pos    =      7.0 : 1.0\n",
      "        contains(shoddy) = True              neg : pos    =      7.0 : 1.0\n",
      "        contains(suvari) = True              neg : pos    =      7.0 : 1.0\n"
     ]
    }
   ],
   "source": [
    "classifier.show_most_informative_features(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 词性分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['e',\n",
       " 'he',\n",
       " 'the',\n",
       " 'n',\n",
       " 'on',\n",
       " 'ton',\n",
       " 'y',\n",
       " 'ty',\n",
       " 'nty',\n",
       " 'd',\n",
       " 'nd',\n",
       " 'and',\n",
       " 'ry',\n",
       " 'ury',\n",
       " 'id',\n",
       " 'aid',\n",
       " 'ay',\n",
       " 'day',\n",
       " 'an',\n",
       " 'ion',\n",
       " 'f',\n",
       " 'of',\n",
       " 's',\n",
       " \"'s\",\n",
       " \"a's\",\n",
       " 't',\n",
       " 'nt',\n",
       " 'ent',\n",
       " 'ary',\n",
       " 'ed',\n",
       " 'ced',\n",
       " '`',\n",
       " '``',\n",
       " 'o',\n",
       " 'no',\n",
       " 'ce',\n",
       " 'nce',\n",
       " \"'\",\n",
       " \"''\",\n",
       " 'at',\n",
       " 'hat',\n",
       " 'ny',\n",
       " 'any',\n",
       " 'es',\n",
       " 'ies',\n",
       " 'k',\n",
       " 'ok',\n",
       " 'ook',\n",
       " 'ace',\n",
       " '.',\n",
       " 'r',\n",
       " 'er',\n",
       " 'her',\n",
       " 'in',\n",
       " 'end',\n",
       " 'ts',\n",
       " 'nts',\n",
       " 'ity',\n",
       " 've',\n",
       " 'ive',\n",
       " 'ee',\n",
       " 'tee',\n",
       " ',',\n",
       " 'h',\n",
       " 'ch',\n",
       " 'ich',\n",
       " 'ad',\n",
       " 'had',\n",
       " 'l',\n",
       " 'll',\n",
       " 'all',\n",
       " 'ge',\n",
       " 'rge',\n",
       " 'ves',\n",
       " 'se',\n",
       " 'ise',\n",
       " 'ks',\n",
       " 'nks',\n",
       " 'a',\n",
       " 'ta',\n",
       " 'nta',\n",
       " 'or',\n",
       " 'for',\n",
       " 'ner',\n",
       " 'as',\n",
       " 'was',\n",
       " 'ted',\n",
       " 'ber',\n",
       " 'm',\n",
       " 'rm',\n",
       " 'erm',\n",
       " 'en',\n",
       " 'een',\n",
       " 'ged',\n",
       " 'by',\n",
       " 'ior',\n",
       " 'rt',\n",
       " 'urt',\n",
       " 'dge',\n",
       " 'od']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.corpus import brown\n",
    "\n",
    "# 找出最常见的后缀\n",
    "suffix_fdist = nltk.FreqDist()\n",
    "for word in brown.words():\n",
    "    word = word.lower()\n",
    "    suffix_fdist[word[-1:]] += 1\n",
    "    suffix_fdist[word[-2:]] += 1\n",
    "    suffix_fdist[word[-3:]] += 1\n",
    "    \n",
    "common_suffixes = list(suffix_fdist.keys())[:100]\n",
    "common_suffixes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 词性特征提取器\n",
    "def pos_features(word):\n",
    "    features = {}\n",
    "    for suffix in common_suffixes:\n",
    "        features['endswith(%s)' % suffix] = word.lower().endswith(suffix)\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5689706613625062"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagged_words = brown.tagged_words(categories='news')\n",
    "featuresets = [(pos_features(n), g) for (n, g) in tagged_words]\n",
    "\n",
    "size = int(len(featuresets) * 0.1)\n",
    "train_set, test_set = featuresets[size:], featuresets[:size]\n",
    "\n",
    "classifier = nltk.DecisionTreeClassifier.train(train_set)\n",
    "nltk.classify.accuracy(classifier, test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'NNS'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier.classify(pos_features('cats'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 探索上下文语境"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 基于句子的词特征提取\n",
    "# 输入分别为：句链表，句中词的索引\n",
    "# 输出为特征\n",
    "def pos_features(sentence, i):\n",
    "    features = {'suffix(1)': sentence[i][-1:],\n",
    "               'suffix(2)': sentence[i][-2:],\n",
    "               'suffix(3)': sentence[i][-3:]}\n",
    "    if i == 0:\n",
    "        features['prev-word'] = '<START>'\n",
    "    else:\n",
    "        features['prev-word'] = sentence[i - 1]\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['The',\n",
       " 'Fulton',\n",
       " 'County',\n",
       " 'Grand',\n",
       " 'Jury',\n",
       " 'said',\n",
       " 'Friday',\n",
       " 'an',\n",
       " 'investigation',\n",
       " 'of',\n",
       " \"Atlanta's\",\n",
       " 'recent',\n",
       " 'primary',\n",
       " 'election',\n",
       " 'produced',\n",
       " '``',\n",
       " 'no',\n",
       " 'evidence',\n",
       " \"''\",\n",
       " 'that',\n",
       " 'any',\n",
       " 'irregularities',\n",
       " 'took',\n",
       " 'place',\n",
       " '.']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "brown.sents()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'suffix(1)': 'n', 'suffix(2)': 'on', 'suffix(3)': 'ion', 'prev-word': 'an'}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_features(brown.sents()[0], 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7891596220785678"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagged_sents = brown.tagged_sents(categories='news')\n",
    "featuresets = []\n",
    "for tagged_sent in tagged_sents:\n",
    "    untagged_sent = nltk.tag.untag(tagged_sent)\n",
    "    for i, (word, tag) in enumerate(tagged_sent):\n",
    "        featuresets.append((pos_features(untagged_sent, i), tag))\n",
    "        \n",
    "size = int(len(featuresets) * 0.1)\n",
    "train_set, test_set = featuresets[size:], featuresets[:size]\n",
    "\n",
    "classifier = nltk.NaiveBayesClassifier.train(train_set)\n",
    "nltk.classify.accuracy(classifier, test_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 序列分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pos_features(sentence, i, history):\n",
    "    features = {'suffix(1)': sentence[i][-1:],\n",
    "               'suffix(2)': sentence[i][-2:],\n",
    "               'suffix(3)': sentence[i][-3:]}\n",
    "    if i == 0:\n",
    "        features['prev-word'] = '<START>'\n",
    "        features['prev_tag'] = '<START>'\n",
    "    else:\n",
    "        features['prev-word'] = sentence[i - 1]\n",
    "        features['prev_tag'] = history[i - 1]\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 序列分类器\n",
    "class ConsecutivePosTagger(nltk.TaggerI):\n",
    "    def __init__(self, train_sents):\n",
    "        train_set = []\n",
    "        for tagged_sent in train_sents:\n",
    "            untagged_sent = nltk.tag.untag(tagged_sent)\n",
    "            history = []\n",
    "            for i, (word, tag) in enumerate(tagged_sent):\n",
    "                featureset = pos_features(untagged_sent, i, history)\n",
    "                train_set.append((featureset, tag))\n",
    "                history.append(tag)\n",
    "        self.classifier = nltk.NaiveBayesClassifier.train(train_set)\n",
    "    \n",
    "    def tag(self, sentence):\n",
    "        history = []\n",
    "        for i, word in enumerate(sentence):\n",
    "            featureset = pos_features(sentence, i, history)\n",
    "            tag = self.classifier.classify(featureset)\n",
    "            history.append(tag)\n",
    "        return zip(sentence, history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7980528511821975"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagged_sents = brown.tagged_sents(categories='news')\n",
    "\n",
    "size = int(len(tagged_sents) * 0.1)\n",
    "train_sents, test_sents = tagged_sents[size:], tagged_sents[:size]\n",
    "\n",
    "tagger = ConsecutivePosTagger(train_sents)\n",
    "tagger.evaluate(test_sents)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 句子分割"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "sents = nltk.corpus.treebank_raw.sents()\n",
    "tokens = []  # 存储句子\n",
    "boundaries = set()  # 对应句子的词索引\n",
    "offset = 0  # 总词数\n",
    "for sent in nltk.corpus.treebank_raw.sents():\n",
    "    tokens.extend(sent)\n",
    "    offset += len(sent)\n",
    "    boundaries.add(offset - 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['.', 'START'], ['Pierre', 'Vinken', ',', '61', 'years', 'old', ',', 'will', 'join', 'the', 'board', 'as', 'a', 'nonexecutive', 'director', 'Nov', '.', '29', '.'], ...]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def punct_features(tokens, i):\n",
    "    return {'next-word-capitalized': tokens[i + 1][0].isupper(),\n",
    "           'prevword': tokens[i - 1].lower(),\n",
    "           'punct': tokens[i],\n",
    "           'prev-word-is-one-char': len(tokens[i - 1]) == 1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.936026936026936"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 提取可能是句子结束符的特征\n",
    "# 特征 ： 句索引链表\n",
    "featuresets = [(punct_features(tokens, i), (i in boundaries))\n",
    "              for i in range(1, len(tokens) - 1)\n",
    "              if tokens[i] in '.?!']\n",
    "\n",
    "size = int(len(featuresets) * 0.1)\n",
    "train_set, test_set = featuresets[size:], featuresets[:size]\n",
    "\n",
    "classifier = nltk.NaiveBayesClassifier.train(train_set)\n",
    "nltk.classify.accuracy(classifier, test_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 识别对话行为类型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "posts = nltk.corpus.nps_chat.xml_posts()[:10000]\n",
    "\n",
    "def dialogue_act_features(post):\n",
    "    features = {}\n",
    "    for word in nltk.word_tokenize(post):\n",
    "        features['contains(%s)' % word.lower()] = True\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Statement'"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "posts[0].get('class')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.667"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features = [(dialogue_act_features(post.text), post.get('class'))\n",
    "           for post in posts]\n",
    "\n",
    "size = int(len(features) * 0.1)\n",
    "train_set, test_set = features[size:], features[:size]\n",
    "\n",
    "classifier = nltk.NaiveBayesClassifier.train(train_set)\n",
    "nltk.classify.accuracy(classifier, test_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 识别文字蕴含（RTE）\n",
    "了解"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rte_features(rtepair):\n",
    "    extractor = nltk.RTEFeatureExtractor(rtepair)\n",
    "    features = {}\n",
    "    features['word_overlap'] = len(extractor.overlap('word'))\n",
    "    features['word_hyp_extra'] = len(extractor.hyp_extra('word'))\n",
    "    features['ne_overlap'] = len(extractor.overlap('ne'))\n",
    "    features['ne_hyp_extra'] = len(extractor.hyp_extra('ne'))\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Asia',\n",
       " 'China',\n",
       " 'Co',\n",
       " 'Davudi',\n",
       " 'Iran',\n",
       " 'Organisation',\n",
       " 'Parviz',\n",
       " 'Russia',\n",
       " 'SCO',\n",
       " 'Shanghai',\n",
       " 'Soviet',\n",
       " 'association',\n",
       " 'at',\n",
       " 'binds',\n",
       " 'central',\n",
       " 'fight',\n",
       " 'fledgling',\n",
       " 'former',\n",
       " 'four',\n",
       " 'meeting',\n",
       " 'operation',\n",
       " 'representing',\n",
       " 'republics',\n",
       " 'terrorism.',\n",
       " 'that',\n",
       " 'together',\n",
       " 'was'}"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rtepair = nltk.corpus.rte.pairs(['rte3_dev.xml'])[33]\n",
    "extrator = nltk.RTEFeatureExtractor(rtepair)\n",
    "extrator.text_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'China', 'SCO.', 'member'}"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extrator.hyp_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extrator.overlap('word')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'China'}"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extrator.overlap('ne')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'member'}"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extrator.hyp_extra('word')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
